"""
数据集简介:https://zhuanlan.zhihu.com/p/365093156
"""
from sklearn.preprocessing import MinMaxScaler
from sklearn.feature_selection import chi2
import pandas as pd


def get_data(path, nans=True):
    data_source = pd.read_csv(path)
    if nans == False:
        return data_source.dropna(axis='index')
    return data_source


def data_pretreat():
    data_source = get_data('../resource/all.csv', nans=False)
    # 删除无用字段
    bad_columns = ['SourceIP', 'DestinationIP', 'SourcePort', 'DestinationPort', 'TimeStamp']
    data_source.drop(labels=bad_columns, axis='columns', inplace=True)
    # 字符数据数字化(这里恶心,不能写成'True',读进来自动转为布尔值True)
    data_source['DoH'] = data_source['DoH'].map({True: 0, False: 1})
    # 归一化处理
    scaler = MinMaxScaler()
    return pd.DataFrame(scaler.fit_transform(data_source), columns=data_source.columns)


if __name__ == '__main__':
    data = data_pretreat()
    print(data.head())
    X = data.loc[:, data.columns != 'DoH']
    Y = data['DoH']
    # print(X)
    # print(Y.value_counts())
    # 卡方检验 https://www.jianshu.com/p/64974c4de9d4
    chi_scores = chi2(X, Y)
    print(chi_scores)
    p_values = pd.Series(data=chi_scores[0]+chi_scores[1], index=X.columns)
    p_values.sort_values(ascending=True, inplace=True)
    print('Values in order of ascending p-values (lower=more significant)')
    print(p_values)

